# coding=utf-8
import codecs
import fnmatch
import json

import os

import sys

from core.m_logging import logger
from spiders import amazonAnsiListingCoreSpider
from spiders.nodeSpider import NodeSpider

if len(sys.argv) >= 5:
    task_no = sys.argv[1]
    failed_node_dir = sys.argv[2]
    batch_no = sys.argv[3]
    state_dir = sys.argv[4]
else:
    task_no = "default_task"
    failed_node_dir = "../failed_nodes"
    batch_no = "default_batch"
    state_dir = "../failed_nodes_state"

# failed_node_dir = "./failed_nodes"
tasks = {}
task_files = fnmatch.filter(os.listdir(failed_node_dir), 'node*.json')
for f in task_files:
    with codecs.open("%s/%s" % (failed_node_dir, f), mode='r', encoding='utf-8') as opened_file:
        r = json.load(opened_file)
        current_node_no = r["node_no"]
        page_no_list = r["failed_pages"]
        if len(page_no_list) > 0:
            tasks[current_node_no] = page_no_list

# task_no = "default_task"
# batch_no = "default_batch"
nodeSpider = NodeSpider(batch_no, task_no)

# set module variable after import
amazonAnsiListingCoreSpider.task_state_dir = state_dir
amazonAnsiListingCoreSpider.batch_no = batch_no
amazonAnsiListingCoreSpider.task_no = task_no

for t in tasks.items():
    node_no = t[0]
    page_no_list = t[1]
    amazonAnsiListingCoreSpider.scrape(node_no, page_no_list)

logger.info("process is completed...")